#include "InvertedIndex.h"
#include "Mylogger.h"
#include "tinyxml2.h"
#include <math.h>
#include <fstream>
#include <sstream>
#include <map>

using std::cout;
using std::endl;
using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::map;
using namespace tinyxml2;

//获得停用词集合  
void InvertedIndex::buildStopSet(){
    /* std::cout << "buildStopSet :" <<std::endl; */
    for(string filename:_conf._stopFiles){
        //打开文件
        ifstream ifs(filename);
        if(!ifs.good()){
            std::cout << "open " << filename << " fail "<<std::endl;
            return ;
        }
        //读出一个单词
        string word;
        while(ifs >> word){
            //加入set
            _stopWords.insert(word);
        }
        //关闭文件
        ifs.close();
    }
}   

//得到文章的位置偏移库
void InvertedIndex::buildOffsetLib(){
    //1.打开文件
    ifstream ifs(_noDupIndexName);
    //2.循环读取文件的每一行
    string line;
    while(getline(ifs,line)){
        int docid=0,pos=0,length=0;
        istringstream iss(line);
        iss >> docid >> pos >> length;
        //加入_offsetLib
        _offsetLib[docid] = std::make_pair(pos,length);
    }
    //3.关闭文件
    ifs.close();
}

//由docid获得文章
void InvertedIndex::getWebPage(int docid,string &paper){
    //1.获得起始位置和偏移量
    auto it = _offsetLib.find(docid);
    int pos = it->second.first;
    int length = it->second.second;
    /* cout << "_offsetLib lenth = " << length << "\n"; */
    //1.打开文件
    ifstream ifs(_cleanedPageName);
    //2.偏移到起始位置
    ifs.seekg(pos);
    //3.读取指定字节的信息
    int count = 0;
    string gPaper;
    char buf[4096];
    while(count != length){
        memset(buf,0,sizeof(buf));
        if(length-count > (int)sizeof(buf)-1){
            ifs.read(buf,(int)sizeof(buf)-1);
            gPaper = gPaper + buf;
            /* cout << "strlen(buf) = " << strlen(buf) << endl; */
            count = gPaper.size();
        }else{
            ifs.read(buf,length-count);
            gPaper = gPaper + buf;
            /* cout << "strlen(buf) = " << strlen(buf) << endl; */
            count = gPaper.size();
        }
    }
    paper = gPaper;
    //4.关闭文件
    ifs.close();
}

void InvertedIndex::parsePaper(string paper,WebPage& webPage){
    //载入文件
    XMLDocument doc;
    doc.Parse(paper.c_str(),paper.size());
    XMLElement* docNode = doc.FirstChildElement("doc");
    if(docNode == nullptr){
        LogWarn("Paper parsing error!");
        return ;
    }

    XMLElement * docidNode = docNode->FirstChildElement("docid");
    XMLElement * titleNode = docNode->FirstChildElement("title");
    XMLElement * linkNode = docNode->FirstChildElement("link");
    XMLElement * contentNode = docNode->FirstChildElement("content");
    if(docidNode == nullptr || titleNode==nullptr
       ||linkNode == nullptr || contentNode == nullptr){
        LogWarn("Paper parsing error!");
        return ;
    }

    string docid_string  = docidNode->GetText();
    int docid = std::stoi(docid_string);
    string title  = titleNode->GetText();
    string link  = linkNode->GetText();
    string content  = contentNode->GetText();

    webPage._docId = docid;
    webPage._docUrl = link;
    webPage._docTitle = title;
    webPage._docContent = content;
}

//向数据成员的容器中增加信息
void InvertedIndex::buildPreInfo(WebPage webPage){
    //1.切割文章获得单词
    int docid = webPage._docId;
    string content = webPage._docContent;
    //对数据进行清洗(标点-->空格,'\r'-->空格)
    for(int idx =0;idx <(int)content.size();++idx){
        if(content[idx]=='\r'||ispunct(content[idx])){
            content[idx] = ' ';
        }
    }
    //对清洗完的数据进行切割,将单词和词频加入unordered_map中
    vector<string> words;
    _tool->cut(content,words);
    //2.统计单词出现在此文章中的次数
    for(const string& word:words){
        //如果单词没有出现在停止中，就加入unordered_map
        int count = _stopWords.count(word);
        if(count == 0){
            auto& Map = _TF[docid]; //找到map
            Map[word]++;
            _DF[word]++;
        }
    }
}

void InvertedIndex::buildInvertedIndex(){
    //1.找到文章
    int N = _offsetLib.size();
    for(auto doc: _TF){
        double sumWeight2 = 0;      
        map<string,double> wordToweight; //记录单词和初始权值的对应关系
        for(auto & elem : doc.second){
            //2.从文章中找到单词
            string word = elem.first;
            int tf = elem.second;
            //3.计算权重
            int df = _DF[word];
            double weight = log2(N/(df+1)+1)*tf;
            sumWeight2 += weight*weight;
            wordToweight[word] = weight;
        }
        //将一片文章中的权重归一化处理
        double sumWeight = sqrt(sumWeight2);
        for(auto & elem : wordToweight){
            //从文章中找到单词
            string word = elem.first;
            double endWeight = elem.second/sumWeight;
            //4.插入倒排序列表
            _invertIndex[word].insert(std::make_pair(doc.first,endWeight)); 
        }
    }
}

void InvertedIndex::store(){ //将倒排索引存入磁盘
    //把_index存入磁盘中
    //1.打开文件
    ofstream ofs(_invertIndexName);
    //2.将_index的内容写入磁盘
    for(const pair<string,set<pair<int,double>>>& elem:_invertIndex){
        ofs << elem.first << " ";
        for(const pair<int,double> spair:elem.second){
            ofs<< spair.first<< " "<<spair.second<<" ";
        }
        ofs << "\n";
    }
    //3.关闭文件
    ofs.close();
}

void InvertedIndex::start(){
    //1.准备工作(获得停用词,偏移库)
    LogInfo("Configuration file reading...");
    _conf.buildConf();
    buildStopSet();
    buildOffsetLib();
    //2.读取预备信息
    LogInfo("The inverted index is being created...");
    for(auto offset:_offsetLib){
        int docid = offset.first;
        string paper;
        WebPage webPage;
        getWebPage(docid,paper);
        /* std::cout << "docid = " << webPage._docId << "\n"; */
        /* std::cout << "paper.length = " << paper.size() << "\n"; */
        parsePaper(paper,webPage);
        buildPreInfo(webPage);
    }
    //3.创建倒排索引
    LogInfo("Create an inverted index");
    buildInvertedIndex();
    //4.存储
    store();
    LogInfo("The inverted index is created successfully");
}
